


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.nn.modules.module &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 

  
  <script src="../../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../../torch.html">torch</a> &gt;</li>
        
      <li>torch.nn.modules.module</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.nn.modules.module</h1><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span><span class="p">,</span> <span class="n">namedtuple</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">itertools</span>
<span class="kn">import</span> <span class="nn">weakref</span>
<span class="kn">import</span> <span class="nn">warnings</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">..parameter</span> <span class="kn">import</span> <span class="n">Parameter</span>
<span class="kn">import</span> <span class="nn">torch.utils.hooks</span> <span class="k">as</span> <span class="nn">hooks</span>

<span class="k">class</span> <span class="nc">_IncompatibleKeys</span><span class="p">(</span><span class="n">namedtuple</span><span class="p">(</span><span class="s1">&#39;IncompatibleKeys&#39;</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;missing_keys&#39;</span><span class="p">,</span> <span class="s1">&#39;unexpected_keys&#39;</span><span class="p">])):</span>
    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">missing_keys</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">unexpected_keys</span><span class="p">:</span>
            <span class="k">return</span> <span class="s1">&#39;&lt;All keys matched successfully&gt;&#39;</span>
        <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">_IncompatibleKeys</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__repr__</span><span class="p">()</span>

    <span class="fm">__str__</span> <span class="o">=</span> <span class="fm">__repr__</span>


<span class="k">def</span> <span class="nf">_addindent</span><span class="p">(</span><span class="n">s_</span><span class="p">,</span> <span class="n">numSpaces</span><span class="p">):</span>
    <span class="n">s</span> <span class="o">=</span> <span class="n">s_</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="c1"># don&#39;t do anything for single-line stuff</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">s_</span>
    <span class="n">first</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
    <span class="n">s</span> <span class="o">=</span> <span class="p">[(</span><span class="n">numSpaces</span> <span class="o">*</span> <span class="s1">&#39; &#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">line</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">s</span><span class="p">]</span>
    <span class="n">s</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
    <span class="n">s</span> <span class="o">=</span> <span class="n">first</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span> <span class="o">+</span> <span class="n">s</span>
    <span class="k">return</span> <span class="n">s</span>


<div class="viewcode-block" id="Module"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module">[docs]</a><span class="k">class</span> <span class="nc">Module</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Base class for all neural network modules.</span>

<span class="sd">    Your models should also subclass this class.</span>

<span class="sd">    Modules can also contain other Modules, allowing to nest them in</span>
<span class="sd">    a tree structure. You can assign the submodules as regular attributes::</span>

<span class="sd">        import torch.nn as nn</span>
<span class="sd">        import torch.nn.functional as F</span>

<span class="sd">        class Model(nn.Module):</span>
<span class="sd">            def __init__(self):</span>
<span class="sd">                super(Model, self).__init__()</span>
<span class="sd">                self.conv1 = nn.Conv2d(1, 20, 5)</span>
<span class="sd">                self.conv2 = nn.Conv2d(20, 20, 5)</span>

<span class="sd">            def forward(self, x):</span>
<span class="sd">                x = F.relu(self.conv1(x))</span>
<span class="sd">                return F.relu(self.conv2(x))</span>

<span class="sd">    Submodules assigned in this way will be registered, and will have their</span>
<span class="sd">    parameters converted too when you call :meth:`to`, etc.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">dump_patches</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;This allows better BC support for :meth:`load_state_dict`. In</span>
<span class="sd">    :meth:`state_dict`, the version number will be saved as in the attribute</span>
<span class="sd">    `_metadata` of the returned state dict, and thus pickled. `_metadata` is a</span>
<span class="sd">    dictionary with keys that follow the naming convention of state dict. See</span>
<span class="sd">    ``_load_from_state_dict`` on how to use this information in loading.</span>

<span class="sd">    If new parameters/buffers are added/removed from a module, this number shall</span>
<span class="sd">    be bumped, and the module&#39;s `_load_from_state_dict` method can compare the</span>
<span class="sd">    version number and do appropriate changes if the state dict is from before</span>
<span class="sd">    the change.&quot;&quot;&quot;</span>
    <span class="n">_version</span> <span class="o">=</span> <span class="mi">1</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Initializes internal Module state, shared by both nn.Module and ScriptModule.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_log_api_usage_once</span><span class="p">(</span><span class="s2">&quot;python.nn_module&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">training</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_backward_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_forward_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_forward_pre_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_state_dict_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_load_state_dict_pre_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>

<div class="viewcode-block" id="Module.forward"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="nb">input</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines the computation performed at every call.</span>

<span class="sd">        Should be overridden by all subclasses.</span>

<span class="sd">        .. note::</span>
<span class="sd">            Although the recipe for forward pass needs to be defined within</span>
<span class="sd">            this function, one should call the :class:`Module` instance afterwards</span>
<span class="sd">            instead of this since the former takes care of running the</span>
<span class="sd">            registered hooks while the latter silently ignores them.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span></div>

<div class="viewcode-block" id="Module.register_buffer"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.register_buffer">[docs]</a>    <span class="k">def</span> <span class="nf">register_buffer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">tensor</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Adds a persistent buffer to the module.</span>

<span class="sd">        This is typically used to register a buffer that should not to be</span>
<span class="sd">        considered a model parameter. For example, BatchNorm&#39;s ``running_mean``</span>
<span class="sd">        is not a parameter, but is part of the persistent state.</span>

<span class="sd">        Buffers can be accessed as attributes using given names.</span>

<span class="sd">        Args:</span>
<span class="sd">            name (string): name of the buffer. The buffer can be accessed</span>
<span class="sd">                from this module using the given name</span>
<span class="sd">            tensor (Tensor): buffer to be registered.</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; self.register_buffer(&#39;running_mean&#39;, torch.zeros(num_features))</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="s1">&#39;_buffers&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
                <span class="s2">&quot;cannot assign buffer before Module.__init__() call&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_six</span><span class="o">.</span><span class="n">string_classes</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;buffer name should be a string. &quot;</span>
                            <span class="s2">&quot;Got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">name</span><span class="p">)))</span>
        <span class="k">elif</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">name</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;buffer name can&#39;t contain </span><span class="se">\&quot;</span><span class="s2">.</span><span class="se">\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;buffer name can&#39;t be empty string </span><span class="se">\&quot;\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;attribute &#39;</span><span class="si">{}</span><span class="s2">&#39; already exists&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
        <span class="k">elif</span> <span class="n">tensor</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;cannot assign &#39;</span><span class="si">{}</span><span class="s2">&#39; object to buffer &#39;</span><span class="si">{}</span><span class="s2">&#39; &quot;</span>
                            <span class="s2">&quot;(torch Tensor or None required)&quot;</span>
                            <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">tensor</span><span class="p">),</span> <span class="n">name</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">tensor</span></div>

<div class="viewcode-block" id="Module.register_parameter"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.register_parameter">[docs]</a>    <span class="k">def</span> <span class="nf">register_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Adds a parameter to the module.</span>

<span class="sd">        The parameter can be accessed as an attribute using given name.</span>

<span class="sd">        Args:</span>
<span class="sd">            name (string): name of the parameter. The parameter can be accessed</span>
<span class="sd">                from this module using the given name</span>
<span class="sd">            param (Parameter): parameter to be added to the module.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="s1">&#39;_parameters&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
                <span class="s2">&quot;cannot assign parameter before Module.__init__() call&quot;</span><span class="p">)</span>

        <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_six</span><span class="o">.</span><span class="n">string_classes</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;parameter name should be a string. &quot;</span>
                            <span class="s2">&quot;Got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">name</span><span class="p">)))</span>
        <span class="k">elif</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">name</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;parameter name can&#39;t contain </span><span class="se">\&quot;</span><span class="s2">.</span><span class="se">\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;parameter name can&#39;t be empty string </span><span class="se">\&quot;\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;attribute &#39;</span><span class="si">{}</span><span class="s2">&#39; already exists&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>

        <span class="k">if</span> <span class="n">param</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;cannot assign &#39;</span><span class="si">{}</span><span class="s2">&#39; object to parameter &#39;</span><span class="si">{}</span><span class="s2">&#39; &quot;</span>
                            <span class="s2">&quot;(torch.nn.Parameter or None required)&quot;</span>
                            <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">param</span><span class="p">),</span> <span class="n">name</span><span class="p">))</span>
        <span class="k">elif</span> <span class="n">param</span><span class="o">.</span><span class="n">grad_fn</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;Cannot assign non-leaf Tensor to parameter &#39;</span><span class="si">{0}</span><span class="s2">&#39;. Model &quot;</span>
                <span class="s2">&quot;parameters must be created explicitly. To express &#39;</span><span class="si">{0}</span><span class="s2">&#39; &quot;</span>
                <span class="s2">&quot;as a function of another Tensor, compute the value in &quot;</span>
                <span class="s2">&quot;the forward() method.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">param</span></div>

<div class="viewcode-block" id="Module.add_module"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.add_module">[docs]</a>    <span class="k">def</span> <span class="nf">add_module</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Adds a child module to the current module.</span>

<span class="sd">        The module can be accessed as an attribute using the given name.</span>

<span class="sd">        Args:</span>
<span class="sd">            name (string): name of the child module. The child module can be</span>
<span class="sd">                accessed from this module using the given name</span>
<span class="sd">            module (Module): child module to be added to the module.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">Module</span><span class="p">)</span> <span class="ow">and</span> <span class="n">module</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> is not a Module subclass&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">module</span><span class="p">)))</span>
        <span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_six</span><span class="o">.</span><span class="n">string_classes</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;module name should be a string. Got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">name</span><span class="p">)))</span>
        <span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;attribute &#39;</span><span class="si">{}</span><span class="s2">&#39; already exists&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
        <span class="k">elif</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">name</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;module name can&#39;t contain </span><span class="se">\&quot;</span><span class="s2">.</span><span class="se">\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;module name can&#39;t be empty string </span><span class="se">\&quot;\&quot;</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">module</span></div>

    <span class="k">def</span> <span class="nf">_apply</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fn</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">children</span><span class="p">():</span>
            <span class="n">module</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>

        <span class="k">def</span> <span class="nf">compute_should_use_set_data</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">tensor_applied</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">_has_compatible_shallow_copy_type</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">tensor_applied</span><span class="p">):</span>
                <span class="c1"># If the new tensor has compatible tensor type as the existing tensor,</span>
                <span class="c1"># the current behavior is to change the tensor in-place using `.data =`,</span>
                <span class="c1"># and the future behavior is to overwrite the existing tensor. However,</span>
                <span class="c1"># changing the current behavior is a BC-breaking change, and we want it</span>
                <span class="c1"># to happen in future releases. So for now we introduce the</span>
                <span class="c1"># `torch.__future__.get_overwrite_module_params_on_conversion()`</span>
                <span class="c1"># global flag to let the user control whether they want the future</span>
                <span class="c1"># behavior of overwriting the existing tensor or not.</span>
                <span class="k">return</span> <span class="ow">not</span> <span class="n">torch</span><span class="o">.</span><span class="n">__future__</span><span class="o">.</span><span class="n">get_overwrite_module_params_on_conversion</span><span class="p">()</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">return</span> <span class="kc">False</span>

        <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">param</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="c1"># Tensors stored in modules are graph leaves, and we don&#39;t want to</span>
                <span class="c1"># track autograd history of `param_applied`, so we have to use</span>
                <span class="c1"># `with torch.no_grad():`</span>
                <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
                    <span class="n">param_applied</span> <span class="o">=</span> <span class="n">fn</span><span class="p">(</span><span class="n">param</span><span class="p">)</span>
                <span class="n">should_use_set_data</span> <span class="o">=</span> <span class="n">compute_should_use_set_data</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="n">param_applied</span><span class="p">)</span>
                <span class="k">if</span> <span class="n">should_use_set_data</span><span class="p">:</span>
                    <span class="n">param</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">param_applied</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">)</span>
                    <span class="k">assert</span> <span class="n">param</span><span class="o">.</span><span class="n">is_leaf</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">param_applied</span><span class="p">,</span> <span class="n">param</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">)</span>

                <span class="k">if</span> <span class="n">param</span><span class="o">.</span><span class="n">grad</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
                        <span class="n">grad_applied</span> <span class="o">=</span> <span class="n">fn</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="p">)</span>
                    <span class="n">should_use_set_data</span> <span class="o">=</span> <span class="n">compute_should_use_set_data</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="p">,</span> <span class="n">grad_applied</span><span class="p">)</span>
                    <span class="k">if</span> <span class="n">should_use_set_data</span><span class="p">:</span>
                        <span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">grad_applied</span>
                    <span class="k">else</span><span class="p">:</span>
                        <span class="k">assert</span> <span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">is_leaf</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="n">grad_applied</span><span class="o">.</span><span class="n">requires_grad_</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">)</span>

        <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">buf</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">buf</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">fn</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>

        <span class="k">return</span> <span class="bp">self</span>

<div class="viewcode-block" id="Module.apply"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.apply">[docs]</a>    <span class="k">def</span> <span class="nf">apply</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fn</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Applies ``fn`` recursively to every submodule (as returned by ``.children()``)</span>
<span class="sd">        as well as self. Typical use includes initializing the parameters of a model</span>
<span class="sd">        (see also :ref:`nn-init-doc`).</span>

<span class="sd">        Args:</span>
<span class="sd">            fn (:class:`Module` -&gt; None): function to be applied to each submodule</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; @torch.no_grad()</span>
<span class="sd">            &gt;&gt;&gt; def init_weights(m):</span>
<span class="sd">            &gt;&gt;&gt;     print(m)</span>
<span class="sd">            &gt;&gt;&gt;     if type(m) == nn.Linear:</span>
<span class="sd">            &gt;&gt;&gt;         m.weight.fill_(1.0)</span>
<span class="sd">            &gt;&gt;&gt;         print(m.weight)</span>
<span class="sd">            &gt;&gt;&gt; net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))</span>
<span class="sd">            &gt;&gt;&gt; net.apply(init_weights)</span>
<span class="sd">            Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 1.,  1.],</span>
<span class="sd">                    [ 1.,  1.]])</span>
<span class="sd">            Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 1.,  1.],</span>
<span class="sd">                    [ 1.,  1.]])</span>
<span class="sd">            Sequential(</span>
<span class="sd">              (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">              (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            )</span>
<span class="sd">            Sequential(</span>
<span class="sd">              (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">              (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            )</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">children</span><span class="p">():</span>
            <span class="n">module</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
        <span class="n">fn</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span></div>

<div class="viewcode-block" id="Module.cuda"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.cuda">[docs]</a>    <span class="k">def</span> <span class="nf">cuda</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Moves all model parameters and buffers to the GPU.</span>

<span class="sd">        This also makes associated parameters and buffers different objects. So</span>
<span class="sd">        it should be called before constructing optimizer if the module will</span>
<span class="sd">        live on GPU while being optimized.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            device (int, optional): if specified, all parameters will be</span>
<span class="sd">                copied to that device</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">cuda</span><span class="p">(</span><span class="n">device</span><span class="p">))</span></div>

<div class="viewcode-block" id="Module.cpu"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.cpu">[docs]</a>    <span class="k">def</span> <span class="nf">cpu</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Moves all model parameters and buffers to the CPU.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">cpu</span><span class="p">())</span></div>

<div class="viewcode-block" id="Module.type"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.type">[docs]</a>    <span class="k">def</span> <span class="nf">type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dst_type</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Casts all parameters and buffers to :attr:`dst_type`.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            dst_type (type or string): the desired type</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">type</span><span class="p">(</span><span class="n">dst_type</span><span class="p">))</span></div>

<div class="viewcode-block" id="Module.float"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.float">[docs]</a>    <span class="k">def</span> <span class="nf">float</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Casts all floating point parameters and buffers to float datatype.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">float</span><span class="p">()</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="n">t</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.double"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.double">[docs]</a>    <span class="k">def</span> <span class="nf">double</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Casts all floating point parameters and buffers to ``double`` datatype.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">double</span><span class="p">()</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="n">t</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.half"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.half">[docs]</a>    <span class="k">def</span> <span class="nf">half</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Casts all floating point parameters and buffers to ``half`` datatype.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">half</span><span class="p">()</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="n">t</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.bfloat16"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.bfloat16">[docs]</a>    <span class="k">def</span> <span class="nf">bfloat16</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Casts all floating point parameters and buffers to ``bfloat16`` datatype.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">bfloat16</span><span class="p">()</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="n">t</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.to"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.to">[docs]</a>    <span class="k">def</span> <span class="nf">to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Moves and/or casts the parameters and buffers.</span>

<span class="sd">        This can be called as</span>

<span class="sd">        .. function:: to(device=None, dtype=None, non_blocking=False)</span>

<span class="sd">        .. function:: to(dtype, non_blocking=False)</span>

<span class="sd">        .. function:: to(tensor, non_blocking=False)</span>

<span class="sd">        .. function:: to(memory_format=torch.channels_last)</span>

<span class="sd">        Its signature is similar to :meth:`torch.Tensor.to`, but only accepts</span>
<span class="sd">        floating point desired :attr:`dtype` s. In addition, this method will</span>
<span class="sd">        only cast the floating point parameters and buffers to :attr:`dtype`</span>
<span class="sd">        (if given). The integral parameters and buffers will be moved</span>
<span class="sd">        :attr:`device`, if that is given, but with dtypes unchanged. When</span>
<span class="sd">        :attr:`non_blocking` is set, it tries to convert/move asynchronously</span>
<span class="sd">        with respect to the host if possible, e.g., moving CPU Tensors with</span>
<span class="sd">        pinned memory to CUDA devices.</span>

<span class="sd">        See below for examples.</span>

<span class="sd">        .. note::</span>
<span class="sd">            This method modifies the module in-place.</span>

<span class="sd">        Args:</span>
<span class="sd">            device (:class:`torch.device`): the desired device of the parameters</span>
<span class="sd">                and buffers in this module</span>
<span class="sd">            dtype (:class:`torch.dtype`): the desired floating point type of</span>
<span class="sd">                the floating point parameters and buffers in this module</span>
<span class="sd">            tensor (torch.Tensor): Tensor whose dtype and device are the desired</span>
<span class="sd">                dtype and device for all parameters and buffers in this module</span>
<span class="sd">            memory_format (:class:`torch.memory_format`): the desired memory</span>
<span class="sd">                format for 4D parameters and buffers in this module (keyword</span>
<span class="sd">                only argument)</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; linear = nn.Linear(2, 2)</span>
<span class="sd">            &gt;&gt;&gt; linear.weight</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 0.1913, -0.3420],</span>
<span class="sd">                    [-0.5113, -0.2325]])</span>
<span class="sd">            &gt;&gt;&gt; linear.to(torch.double)</span>
<span class="sd">            Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            &gt;&gt;&gt; linear.weight</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 0.1913, -0.3420],</span>
<span class="sd">                    [-0.5113, -0.2325]], dtype=torch.float64)</span>
<span class="sd">            &gt;&gt;&gt; gpu1 = torch.device(&quot;cuda:1&quot;)</span>
<span class="sd">            &gt;&gt;&gt; linear.to(gpu1, dtype=torch.half, non_blocking=True)</span>
<span class="sd">            Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            &gt;&gt;&gt; linear.weight</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 0.1914, -0.3420],</span>
<span class="sd">                    [-0.5112, -0.2324]], dtype=torch.float16, device=&#39;cuda:1&#39;)</span>
<span class="sd">            &gt;&gt;&gt; cpu = torch.device(&quot;cpu&quot;)</span>
<span class="sd">            &gt;&gt;&gt; linear.to(cpu)</span>
<span class="sd">            Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            &gt;&gt;&gt; linear.weight</span>
<span class="sd">            Parameter containing:</span>
<span class="sd">            tensor([[ 0.1914, -0.3420],</span>
<span class="sd">                    [-0.5112, -0.2324]], dtype=torch.float16)</span>

<span class="sd">        &quot;&quot;&quot;</span>

        <span class="n">device</span><span class="p">,</span> <span class="n">dtype</span><span class="p">,</span> <span class="n">non_blocking</span><span class="p">,</span> <span class="n">convert_to_format</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_nn</span><span class="o">.</span><span class="n">_parse_to</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">dtype</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">dtype</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;nn.Module.to only accepts floating point &#39;</span>
                                <span class="s1">&#39;dtypes, but got desired dtype=</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dtype</span><span class="p">))</span>

        <span class="k">def</span> <span class="nf">convert</span><span class="p">(</span><span class="n">t</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">convert_to_format</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">t</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">==</span> <span class="mi">4</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">t</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">dtype</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span> <span class="n">non_blocking</span><span class="p">,</span> <span class="n">memory_format</span><span class="o">=</span><span class="n">convert_to_format</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">t</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="n">dtype</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">()</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span> <span class="n">non_blocking</span><span class="p">)</span>

        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="n">convert</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.register_backward_hook"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.register_backward_hook">[docs]</a>    <span class="k">def</span> <span class="nf">register_backward_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Registers a backward hook on the module.</span>

<span class="sd">        The hook will be called every time the gradients with respect to module</span>
<span class="sd">        inputs are computed. The hook should have the following signature::</span>

<span class="sd">            hook(module, grad_input, grad_output) -&gt; Tensor or None</span>

<span class="sd">        The :attr:`grad_input` and :attr:`grad_output` may be tuples if the</span>
<span class="sd">        module has multiple inputs or outputs. The hook should not modify its</span>
<span class="sd">        arguments, but it can optionally return a new gradient with respect to</span>
<span class="sd">        input that will be used in place of :attr:`grad_input` in subsequent</span>
<span class="sd">        computations.</span>

<span class="sd">        Returns:</span>
<span class="sd">            :class:`torch.utils.hooks.RemovableHandle`:</span>
<span class="sd">                a handle that can be used to remove the added hook by calling</span>
<span class="sd">                ``handle.remove()``</span>

<span class="sd">        .. warning ::</span>

<span class="sd">            The current implementation will not have the presented behavior</span>
<span class="sd">            for complex :class:`Module` that perform many operations.</span>
<span class="sd">            In some failure cases, :attr:`grad_input` and :attr:`grad_output` will only</span>
<span class="sd">            contain the gradients for a subset of the inputs and outputs.</span>
<span class="sd">            For such :class:`Module`, you should use :func:`torch.Tensor.register_hook`</span>
<span class="sd">            directly on a specific input or output to get the required gradients.</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_backward_hooks</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_backward_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">handle</span></div>

<div class="viewcode-block" id="Module.register_forward_pre_hook"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.register_forward_pre_hook">[docs]</a>    <span class="k">def</span> <span class="nf">register_forward_pre_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Registers a forward pre-hook on the module.</span>

<span class="sd">        The hook will be called every time before :func:`forward` is invoked.</span>
<span class="sd">        It should have the following signature::</span>

<span class="sd">            hook(module, input) -&gt; None or modified input</span>

<span class="sd">        The hook can modify the input. User can either return a tuple or a</span>
<span class="sd">        single modified value in the hook. We will wrap the value into a tuple</span>
<span class="sd">        if a single value is returned(unless that value is already a tuple).</span>

<span class="sd">        Returns:</span>
<span class="sd">            :class:`torch.utils.hooks.RemovableHandle`:</span>
<span class="sd">                a handle that can be used to remove the added hook by calling</span>
<span class="sd">                ``handle.remove()``</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_forward_pre_hooks</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_forward_pre_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">handle</span></div>

<div class="viewcode-block" id="Module.register_forward_hook"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.register_forward_hook">[docs]</a>    <span class="k">def</span> <span class="nf">register_forward_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Registers a forward hook on the module.</span>

<span class="sd">        The hook will be called every time after :func:`forward` has computed an output.</span>
<span class="sd">        It should have the following signature::</span>

<span class="sd">            hook(module, input, output) -&gt; None or modified output</span>

<span class="sd">        The hook can modify the output. It can modify the input inplace but</span>
<span class="sd">        it will not have effect on forward since this is called after</span>
<span class="sd">        :func:`forward` is called.</span>

<span class="sd">        Returns:</span>
<span class="sd">            :class:`torch.utils.hooks.RemovableHandle`:</span>
<span class="sd">                a handle that can be used to remove the added hook by calling</span>
<span class="sd">                ``handle.remove()``</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_forward_hooks</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_forward_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">handle</span></div>


    <span class="k">def</span> <span class="nf">_slow_forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="n">tracing_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_get_tracing_state</span><span class="p">()</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">tracing_state</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">ScriptMethod</span><span class="p">):</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="n">recording_scopes</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">_trace_module_map</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="n">recording_scopes</span><span class="p">:</span>
            <span class="n">name</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">_trace_module_map</span><span class="p">[</span><span class="bp">self</span><span class="p">]</span> <span class="k">if</span> <span class="bp">self</span> <span class="ow">in</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">_trace_module_map</span> <span class="k">else</span> <span class="kc">None</span>
            <span class="k">if</span> <span class="n">name</span><span class="p">:</span>
                <span class="n">cur_scope_name</span> <span class="o">=</span> <span class="n">tracing_state</span><span class="o">.</span><span class="n">current_scope</span><span class="p">()</span>
                <span class="n">tracing_state</span><span class="o">.</span><span class="n">push_scope</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">recording_scopes</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">finally</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">recording_scopes</span><span class="p">:</span>
                <span class="n">tracing_state</span><span class="o">.</span><span class="n">pop_scope</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">result</span>

    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">hook</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_pre_hooks</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
            <span class="n">result</span> <span class="o">=</span> <span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
                    <span class="n">result</span> <span class="o">=</span> <span class="p">(</span><span class="n">result</span><span class="p">,)</span>
                <span class="nb">input</span> <span class="o">=</span> <span class="n">result</span>
        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_get_tracing_state</span><span class="p">():</span>
            <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_slow_forward</span><span class="p">(</span><span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="o">*</span><span class="nb">input</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">hook</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_hooks</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
            <span class="n">hook_result</span> <span class="o">=</span> <span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">result</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">hook_result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">result</span> <span class="o">=</span> <span class="n">hook_result</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_backward_hooks</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">var</span> <span class="o">=</span> <span class="n">result</span>
            <span class="k">while</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">var</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">var</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
                    <span class="n">var</span> <span class="o">=</span> <span class="nb">next</span><span class="p">((</span><span class="n">v</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">var</span><span class="o">.</span><span class="n">values</span><span class="p">()</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)))</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">var</span> <span class="o">=</span> <span class="n">var</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
            <span class="n">grad_fn</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">grad_fn</span>
            <span class="k">if</span> <span class="n">grad_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">hook</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_backward_hooks</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
                    <span class="n">wrapper</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">hook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
                    <span class="n">functools</span><span class="o">.</span><span class="n">update_wrapper</span><span class="p">(</span><span class="n">wrapper</span><span class="p">,</span> <span class="n">hook</span><span class="p">)</span>
                    <span class="n">grad_fn</span><span class="o">.</span><span class="n">register_hook</span><span class="p">(</span><span class="n">wrapper</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">result</span>

    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>
        <span class="c1"># Support loading old checkpoints that don&#39;t have the following attrs:</span>
        <span class="k">if</span> <span class="s1">&#39;_forward_pre_hooks&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_forward_pre_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="k">if</span> <span class="s1">&#39;_state_dict_hooks&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_state_dict_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="k">if</span> <span class="s1">&#39;_load_state_dict_pre_hooks&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_load_state_dict_pre_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>

    <span class="k">def</span> <span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
        <span class="k">if</span> <span class="s1">&#39;_parameters&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="n">_parameters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s1">&#39;_parameters&#39;</span><span class="p">]</span>
            <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">_parameters</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">if</span> <span class="s1">&#39;_buffers&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="n">_buffers</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s1">&#39;_buffers&#39;</span><span class="p">]</span>
            <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">_buffers</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">_buffers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">if</span> <span class="s1">&#39;_modules&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">:</span>
            <span class="n">modules</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s1">&#39;_modules&#39;</span><span class="p">]</span>
            <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">modules</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">modules</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="s2">&quot;&#39;</span><span class="si">{}</span><span class="s2">&#39; object has no attribute &#39;</span><span class="si">{}</span><span class="s2">&#39;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
            <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>

    <span class="k">def</span> <span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
        <span class="k">def</span> <span class="nf">remove_from</span><span class="p">(</span><span class="o">*</span><span class="n">dicts</span><span class="p">):</span>
            <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">dicts</span><span class="p">:</span>
                <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">d</span><span class="p">:</span>
                    <span class="k">del</span> <span class="n">d</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>

        <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;_parameters&#39;</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">params</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
                    <span class="s2">&quot;cannot assign parameters before Module.__init__() call&quot;</span><span class="p">)</span>
            <span class="n">remove_from</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">register_parameter</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;cannot assign &#39;</span><span class="si">{}</span><span class="s2">&#39; as parameter &#39;</span><span class="si">{}</span><span class="s2">&#39; &quot;</span>
                                <span class="s2">&quot;(torch.nn.Parameter or None expected)&quot;</span>
                                <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="n">name</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">register_parameter</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">modules</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;_modules&#39;</span><span class="p">)</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">Module</span><span class="p">):</span>
                <span class="k">if</span> <span class="n">modules</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
                        <span class="s2">&quot;cannot assign module before Module.__init__() call&quot;</span><span class="p">)</span>
                <span class="n">remove_from</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">)</span>
                <span class="n">modules</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
            <span class="k">elif</span> <span class="n">modules</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">modules</span><span class="p">:</span>
                <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;cannot assign &#39;</span><span class="si">{}</span><span class="s2">&#39; as child module &#39;</span><span class="si">{}</span><span class="s2">&#39; &quot;</span>
                                    <span class="s2">&quot;(torch.nn.Module or None expected)&quot;</span>
                                    <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="n">name</span><span class="p">))</span>
                <span class="n">modules</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">buffers</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;_buffers&#39;</span><span class="p">)</span>
                <span class="k">if</span> <span class="n">buffers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">buffers</span><span class="p">:</span>
                    <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
                        <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;cannot assign &#39;</span><span class="si">{}</span><span class="s2">&#39; as buffer &#39;</span><span class="si">{}</span><span class="s2">&#39; &quot;</span>
                                        <span class="s2">&quot;(torch.Tensor or None expected)&quot;</span>
                                        <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="n">name</span><span class="p">))</span>
                    <span class="n">buffers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="nb">object</span><span class="o">.</span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__delattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">:</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">elif</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">:</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">elif</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="p">:</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="nb">object</span><span class="o">.</span><span class="fm">__delattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_register_state_dict_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;These hooks will be called with arguments: `self`, `state_dict`,</span>
<span class="sd">        `prefix`, `local_metadata`, after the `state_dict` of `self` is set.</span>
<span class="sd">        Note that only parameters and buffers of `self` or its children are</span>
<span class="sd">        guaranteed to exist in `state_dict`. The hooks may modify `state_dict`</span>
<span class="sd">        inplace or return a new one.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_state_dict_hooks</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_state_dict_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">handle</span>

    <span class="k">def</span> <span class="nf">_save_to_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">destination</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">keep_vars</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves module state to `destination` dictionary, containing a state</span>
<span class="sd">        of the module, but not its descendants. This is called on every</span>
<span class="sd">        submodule in :meth:`~torch.nn.Module.state_dict`.</span>

<span class="sd">        In rare cases, subclasses can achieve class-specific behavior by</span>
<span class="sd">        overriding this method with custom logic.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            destination (dict): a dict where state will be stored</span>
<span class="sd">            prefix (str): the prefix for parameters and buffers used in this</span>
<span class="sd">                module</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">param</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">destination</span><span class="p">[</span><span class="n">prefix</span> <span class="o">+</span> <span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">param</span> <span class="k">if</span> <span class="n">keep_vars</span> <span class="k">else</span> <span class="n">param</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">buf</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">buf</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">destination</span><span class="p">[</span><span class="n">prefix</span> <span class="o">+</span> <span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">buf</span> <span class="k">if</span> <span class="n">keep_vars</span> <span class="k">else</span> <span class="n">buf</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>

<div class="viewcode-block" id="Module.state_dict"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">destination</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">keep_vars</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns a dictionary containing a whole state of the module.</span>

<span class="sd">        Both parameters and persistent buffers (e.g. running averages) are</span>
<span class="sd">        included. Keys are corresponding parameter and buffer names.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict:</span>
<span class="sd">                a dictionary containing a whole state of the module</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; module.state_dict().keys()</span>
<span class="sd">            [&#39;bias&#39;, &#39;weight&#39;]</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">destination</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">destination</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
            <span class="n">destination</span><span class="o">.</span><span class="n">_metadata</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="n">destination</span><span class="o">.</span><span class="n">_metadata</span><span class="p">[</span><span class="n">prefix</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span> <span class="o">=</span> <span class="n">local_metadata</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_save_to_state_dict</span><span class="p">(</span><span class="n">destination</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">keep_vars</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">module</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">module</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(</span><span class="n">destination</span><span class="p">,</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="n">keep_vars</span><span class="o">=</span><span class="n">keep_vars</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">hook</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_state_dict_hooks</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
            <span class="n">hook_result</span> <span class="o">=</span> <span class="n">hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">destination</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">local_metadata</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">hook_result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">destination</span> <span class="o">=</span> <span class="n">hook_result</span>
        <span class="k">return</span> <span class="n">destination</span></div>

    <span class="k">def</span> <span class="nf">_register_load_state_dict_pre_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;These hooks will be called with arguments: `state_dict`, `prefix`,</span>
<span class="sd">        `local_metadata`, `strict`, `missing_keys`, `unexpected_keys`,</span>
<span class="sd">        `error_msgs`, before loading `state_dict` into `self`. These arguments</span>
<span class="sd">        are exactly the same as those of `_load_from_state_dict`.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_load_state_dict_pre_hooks</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_load_state_dict_pre_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">handle</span>

    <span class="k">def</span> <span class="nf">_load_from_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">local_metadata</span><span class="p">,</span> <span class="n">strict</span><span class="p">,</span>
                              <span class="n">missing_keys</span><span class="p">,</span> <span class="n">unexpected_keys</span><span class="p">,</span> <span class="n">error_msgs</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Copies parameters and buffers from :attr:`state_dict` into only</span>
<span class="sd">        this module, but not its descendants. This is called on every submodule</span>
<span class="sd">        in :meth:`~torch.nn.Module.load_state_dict`. Metadata saved for this</span>
<span class="sd">        module in input :attr:`state_dict` is provided as :attr:`local_metadata`.</span>
<span class="sd">        For state dicts without metadata, :attr:`local_metadata` is empty.</span>
<span class="sd">        Subclasses can achieve class-specific backward compatible loading using</span>
<span class="sd">        the version number at `local_metadata.get(&quot;version&quot;, None)`.</span>

<span class="sd">        .. note::</span>
<span class="sd">            :attr:`state_dict` is not the same object as the input</span>
<span class="sd">            :attr:`state_dict` to :meth:`~torch.nn.Module.load_state_dict`. So</span>
<span class="sd">            it can be modified.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            state_dict (dict): a dict containing parameters and</span>
<span class="sd">                persistent buffers.</span>
<span class="sd">            prefix (str): the prefix for parameters and buffers used in this</span>
<span class="sd">                module</span>
<span class="sd">            local_metadata (dict): a dict containing the metadata for this module.</span>
<span class="sd">                See</span>
<span class="sd">            strict (bool): whether to strictly enforce that the keys in</span>
<span class="sd">                :attr:`state_dict` with :attr:`prefix` match the names of</span>
<span class="sd">                parameters and buffers in this module</span>
<span class="sd">            missing_keys (list of str): if ``strict=True``, add missing keys to</span>
<span class="sd">                this list</span>
<span class="sd">            unexpected_keys (list of str): if ``strict=True``, add unexpected</span>
<span class="sd">                keys to this list</span>
<span class="sd">            error_msgs (list of str): error messages should be added to this</span>
<span class="sd">                list, and will be reported together in</span>
<span class="sd">                :meth:`~torch.nn.Module.load_state_dict`</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">hook</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_state_dict_pre_hooks</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
            <span class="n">hook</span><span class="p">(</span><span class="n">state_dict</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">local_metadata</span><span class="p">,</span> <span class="n">strict</span><span class="p">,</span> <span class="n">missing_keys</span><span class="p">,</span> <span class="n">unexpected_keys</span><span class="p">,</span> <span class="n">error_msgs</span><span class="p">)</span>

        <span class="n">local_name_params</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">items</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
        <span class="n">local_state</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">local_name_params</span> <span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">}</span>

        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">local_state</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">key</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">name</span>
            <span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">state_dict</span><span class="p">:</span>
                <span class="n">input_param</span> <span class="o">=</span> <span class="n">state_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>

                <span class="c1"># Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+</span>
                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">input_param</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
                    <span class="n">input_param</span> <span class="o">=</span> <span class="n">input_param</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

                <span class="k">if</span> <span class="n">input_param</span><span class="o">.</span><span class="n">shape</span> <span class="o">!=</span> <span class="n">param</span><span class="o">.</span><span class="n">shape</span><span class="p">:</span>
                    <span class="c1"># local shape should match the one in checkpoint</span>
                    <span class="n">error_msgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;size mismatch for </span><span class="si">{}</span><span class="s1">: copying a param with shape </span><span class="si">{}</span><span class="s1"> from checkpoint, &#39;</span>
                                      <span class="s1">&#39;the shape in current model is </span><span class="si">{}</span><span class="s1">.&#39;</span>
                                      <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">input_param</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">param</span><span class="o">.</span><span class="n">shape</span><span class="p">))</span>
                    <span class="k">continue</span>

                <span class="k">try</span><span class="p">:</span>
                    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
                        <span class="n">param</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">input_param</span><span class="p">)</span>
                <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
                    <span class="n">error_msgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;While copying the parameter named &quot;</span><span class="si">{}</span><span class="s1">&quot;, &#39;</span>
                                      <span class="s1">&#39;whose dimensions in the model are </span><span class="si">{}</span><span class="s1"> and &#39;</span>
                                      <span class="s1">&#39;whose dimensions in the checkpoint are </span><span class="si">{}</span><span class="s1">, &#39;</span>
                                      <span class="s1">&#39;an exception occured : </span><span class="si">{}</span><span class="s1">.&#39;</span>
                                      <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">param</span><span class="o">.</span><span class="n">size</span><span class="p">(),</span> <span class="n">input_param</span><span class="o">.</span><span class="n">size</span><span class="p">(),</span> <span class="n">ex</span><span class="o">.</span><span class="n">args</span><span class="p">))</span>
            <span class="k">elif</span> <span class="n">strict</span><span class="p">:</span>
                <span class="n">missing_keys</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">strict</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">state_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
                <span class="k">if</span> <span class="n">key</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">prefix</span><span class="p">):</span>
                    <span class="n">input_name</span> <span class="o">=</span> <span class="n">key</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">prefix</span><span class="p">):]</span>
                    <span class="n">input_name</span> <span class="o">=</span> <span class="n">input_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>  <span class="c1"># get the name of param/buffer/child</span>
                    <span class="k">if</span> <span class="n">input_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span> <span class="ow">and</span> <span class="n">input_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">local_state</span><span class="p">:</span>
                        <span class="n">unexpected_keys</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>

<div class="viewcode-block" id="Module.load_state_dict"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.load_state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Copies parameters and buffers from :attr:`state_dict` into</span>
<span class="sd">        this module and its descendants. If :attr:`strict` is ``True``, then</span>
<span class="sd">        the keys of :attr:`state_dict` must exactly match the keys returned</span>
<span class="sd">        by this module&#39;s :meth:`~torch.nn.Module.state_dict` function.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            state_dict (dict): a dict containing parameters and</span>
<span class="sd">                persistent buffers.</span>
<span class="sd">            strict (bool, optional): whether to strictly enforce that the keys</span>
<span class="sd">                in :attr:`state_dict` match the keys returned by this module&#39;s</span>
<span class="sd">                :meth:`~torch.nn.Module.state_dict` function. Default: ``True``</span>

<span class="sd">        Returns:</span>
<span class="sd">            ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:</span>
<span class="sd">                * **missing_keys** is a list of str containing the missing keys</span>
<span class="sd">                * **unexpected_keys** is a list of str containing the unexpected keys</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">missing_keys</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">unexpected_keys</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">error_msgs</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="c1"># copy state_dict so _load_from_state_dict can modify it</span>
        <span class="n">metadata</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">state_dict</span><span class="p">,</span> <span class="s1">&#39;_metadata&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
        <span class="n">state_dict</span> <span class="o">=</span> <span class="n">state_dict</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="k">if</span> <span class="n">metadata</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">state_dict</span><span class="o">.</span><span class="n">_metadata</span> <span class="o">=</span> <span class="n">metadata</span>

        <span class="k">def</span> <span class="nf">load</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">):</span>
            <span class="n">local_metadata</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">if</span> <span class="n">metadata</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">prefix</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">{})</span>
            <span class="n">module</span><span class="o">.</span><span class="n">_load_from_state_dict</span><span class="p">(</span>
                <span class="n">state_dict</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">local_metadata</span><span class="p">,</span> <span class="kc">True</span><span class="p">,</span> <span class="n">missing_keys</span><span class="p">,</span> <span class="n">unexpected_keys</span><span class="p">,</span> <span class="n">error_msgs</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">child</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                <span class="k">if</span> <span class="n">child</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="n">load</span><span class="p">(</span><span class="n">child</span><span class="p">,</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;.&#39;</span><span class="p">)</span>

        <span class="n">load</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
        <span class="n">load</span> <span class="o">=</span> <span class="kc">None</span>  <span class="c1"># break load-&gt;load reference cycle</span>

        <span class="k">if</span> <span class="n">strict</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">unexpected_keys</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">error_msgs</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
                    <span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;Unexpected key(s) in state_dict: </span><span class="si">{}</span><span class="s1">. &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                        <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">&#39;&quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">unexpected_keys</span><span class="p">)))</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">missing_keys</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">error_msgs</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span>
                    <span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;Missing key(s) in state_dict: </span><span class="si">{}</span><span class="s1">. &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                        <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">&#39;&quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">missing_keys</span><span class="p">)))</span>

        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">error_msgs</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;Error(s) in loading state_dict for </span><span class="si">{}</span><span class="s1">:</span><span class="se">\n\t</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                               <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\n\t</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">error_msgs</span><span class="p">)))</span>
        <span class="k">return</span> <span class="n">_IncompatibleKeys</span><span class="p">(</span><span class="n">missing_keys</span><span class="p">,</span> <span class="n">unexpected_keys</span><span class="p">)</span></div>

    <span class="k">def</span> <span class="nf">_named_members</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">get_members_fn</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Helper method for yielding various names + members of modules.&quot;&quot;&quot;</span>
        <span class="n">memo</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
        <span class="n">modules</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_modules</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">)</span> <span class="k">if</span> <span class="n">recurse</span> <span class="k">else</span> <span class="p">[(</span><span class="n">prefix</span><span class="p">,</span> <span class="bp">self</span><span class="p">)]</span>
        <span class="k">for</span> <span class="n">module_prefix</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">modules</span><span class="p">:</span>
            <span class="n">members</span> <span class="o">=</span> <span class="n">get_members_fn</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">members</span><span class="p">:</span>
                <span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">memo</span><span class="p">:</span>
                    <span class="k">continue</span>
                <span class="n">memo</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
                <span class="n">name</span> <span class="o">=</span> <span class="n">module_prefix</span> <span class="o">+</span> <span class="p">(</span><span class="s1">&#39;.&#39;</span> <span class="k">if</span> <span class="n">module_prefix</span> <span class="k">else</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">k</span>
                <span class="k">yield</span> <span class="n">name</span><span class="p">,</span> <span class="n">v</span>

<div class="viewcode-block" id="Module.parameters"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.parameters">[docs]</a>    <span class="k">def</span> <span class="nf">parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over module parameters.</span>

<span class="sd">        This is typically passed to an optimizer.</span>

<span class="sd">        Args:</span>
<span class="sd">            recurse (bool): if True, then yields parameters of this module</span>
<span class="sd">                and all submodules. Otherwise, yields only parameters that</span>
<span class="sd">                are direct members of this module.</span>

<span class="sd">        Yields:</span>
<span class="sd">            Parameter: module parameter</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; for param in model.parameters():</span>
<span class="sd">            &gt;&gt;&gt;     print(type(param), param.size())</span>
<span class="sd">            &lt;class &#39;torch.Tensor&#39;&gt; (20L,)</span>
<span class="sd">            &lt;class &#39;torch.Tensor&#39;&gt; (20L, 1L, 5L, 5L)</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">(</span><span class="n">recurse</span><span class="o">=</span><span class="n">recurse</span><span class="p">):</span>
            <span class="k">yield</span> <span class="n">param</span></div>

<div class="viewcode-block" id="Module.named_parameters"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.named_parameters">[docs]</a>    <span class="k">def</span> <span class="nf">named_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over module parameters, yielding both the</span>
<span class="sd">        name of the parameter as well as the parameter itself.</span>

<span class="sd">        Args:</span>
<span class="sd">            prefix (str): prefix to prepend to all parameter names.</span>
<span class="sd">            recurse (bool): if True, then yields parameters of this module</span>
<span class="sd">                and all submodules. Otherwise, yields only parameters that</span>
<span class="sd">                are direct members of this module.</span>

<span class="sd">        Yields:</span>
<span class="sd">            (string, Parameter): Tuple containing the name and parameter</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; for name, param in self.named_parameters():</span>
<span class="sd">            &gt;&gt;&gt;    if name in [&#39;bias&#39;]:</span>
<span class="sd">            &gt;&gt;&gt;        print(param.size())</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">gen</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_named_members</span><span class="p">(</span>
            <span class="k">lambda</span> <span class="n">module</span><span class="p">:</span> <span class="n">module</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">items</span><span class="p">(),</span>
            <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="n">recurse</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="n">gen</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">elem</span></div>

<div class="viewcode-block" id="Module.buffers"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.buffers">[docs]</a>    <span class="k">def</span> <span class="nf">buffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over module buffers.</span>

<span class="sd">        Args:</span>
<span class="sd">            recurse (bool): if True, then yields buffers of this module</span>
<span class="sd">                and all submodules. Otherwise, yields only buffers that</span>
<span class="sd">                are direct members of this module.</span>

<span class="sd">        Yields:</span>
<span class="sd">            torch.Tensor: module buffer</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; for buf in model.buffers():</span>
<span class="sd">            &gt;&gt;&gt;     print(type(buf), buf.size())</span>
<span class="sd">            &lt;class &#39;torch.Tensor&#39;&gt; (20L,)</span>
<span class="sd">            &lt;class &#39;torch.Tensor&#39;&gt; (20L, 1L, 5L, 5L)</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">buf</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_buffers</span><span class="p">(</span><span class="n">recurse</span><span class="o">=</span><span class="n">recurse</span><span class="p">):</span>
            <span class="k">yield</span> <span class="n">buf</span></div>

<div class="viewcode-block" id="Module.named_buffers"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.named_buffers">[docs]</a>    <span class="k">def</span> <span class="nf">named_buffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over module buffers, yielding both the</span>
<span class="sd">        name of the buffer as well as the buffer itself.</span>

<span class="sd">        Args:</span>
<span class="sd">            prefix (str): prefix to prepend to all buffer names.</span>
<span class="sd">            recurse (bool): if True, then yields buffers of this module</span>
<span class="sd">                and all submodules. Otherwise, yields only buffers that</span>
<span class="sd">                are direct members of this module.</span>

<span class="sd">        Yields:</span>
<span class="sd">            (string, torch.Tensor): Tuple containing the name and buffer</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; for name, buf in self.named_buffers():</span>
<span class="sd">            &gt;&gt;&gt;    if name in [&#39;running_var&#39;]:</span>
<span class="sd">            &gt;&gt;&gt;        print(buf.size())</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">gen</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_named_members</span><span class="p">(</span>
            <span class="k">lambda</span> <span class="n">module</span><span class="p">:</span> <span class="n">module</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">items</span><span class="p">(),</span>
            <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">,</span> <span class="n">recurse</span><span class="o">=</span><span class="n">recurse</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">elem</span> <span class="ow">in</span> <span class="n">gen</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">elem</span></div>

<div class="viewcode-block" id="Module.children"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.children">[docs]</a>    <span class="k">def</span> <span class="nf">children</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over immediate children modules.</span>

<span class="sd">        Yields:</span>
<span class="sd">            Module: a child module</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_children</span><span class="p">():</span>
            <span class="k">yield</span> <span class="n">module</span></div>

<div class="viewcode-block" id="Module.named_children"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.named_children">[docs]</a>    <span class="k">def</span> <span class="nf">named_children</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over immediate children modules, yielding both</span>
<span class="sd">        the name of the module as well as the module itself.</span>

<span class="sd">        Yields:</span>
<span class="sd">            (string, Module): Tuple containing a name and child module</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; for name, module in model.named_children():</span>
<span class="sd">            &gt;&gt;&gt;     if name in [&#39;conv4&#39;, &#39;conv5&#39;]:</span>
<span class="sd">            &gt;&gt;&gt;         print(module)</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">memo</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">module</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">module</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">memo</span><span class="p">:</span>
                <span class="n">memo</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
                <span class="k">yield</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span></div>

<div class="viewcode-block" id="Module.modules"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.modules">[docs]</a>    <span class="k">def</span> <span class="nf">modules</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over all modules in the network.</span>

<span class="sd">        Yields:</span>
<span class="sd">            Module: a module in the network</span>

<span class="sd">        Note:</span>
<span class="sd">            Duplicate modules are returned only once. In the following</span>
<span class="sd">            example, ``l`` will be returned only once.</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; l = nn.Linear(2, 2)</span>
<span class="sd">            &gt;&gt;&gt; net = nn.Sequential(l, l)</span>
<span class="sd">            &gt;&gt;&gt; for idx, m in enumerate(net.modules()):</span>
<span class="sd">                    print(idx, &#39;-&gt;&#39;, m)</span>

<span class="sd">            0 -&gt; Sequential(</span>
<span class="sd">              (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">              (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            )</span>
<span class="sd">            1 -&gt; Linear(in_features=2, out_features=2, bias=True)</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_modules</span><span class="p">():</span>
            <span class="k">yield</span> <span class="n">module</span></div>

<div class="viewcode-block" id="Module.named_modules"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.named_modules">[docs]</a>    <span class="k">def</span> <span class="nf">named_modules</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns an iterator over all modules in the network, yielding</span>
<span class="sd">        both the name of the module as well as the module itself.</span>

<span class="sd">        Yields:</span>
<span class="sd">            (string, Module): Tuple of name and module</span>

<span class="sd">        Note:</span>
<span class="sd">            Duplicate modules are returned only once. In the following</span>
<span class="sd">            example, ``l`` will be returned only once.</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; l = nn.Linear(2, 2)</span>
<span class="sd">            &gt;&gt;&gt; net = nn.Sequential(l, l)</span>
<span class="sd">            &gt;&gt;&gt; for idx, m in enumerate(net.named_modules()):</span>
<span class="sd">                    print(idx, &#39;-&gt;&#39;, m)</span>

<span class="sd">            0 -&gt; (&#39;&#39;, Sequential(</span>
<span class="sd">              (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">              (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="sd">            ))</span>
<span class="sd">            1 -&gt; (&#39;0&#39;, Linear(in_features=2, out_features=2, bias=True))</span>

<span class="sd">        &quot;&quot;&quot;</span>

        <span class="k">if</span> <span class="n">memo</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">memo</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
        <span class="k">if</span> <span class="bp">self</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">memo</span><span class="p">:</span>
            <span class="n">memo</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
            <span class="k">yield</span> <span class="n">prefix</span><span class="p">,</span> <span class="bp">self</span>
            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                <span class="k">if</span> <span class="n">module</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="k">continue</span>
                <span class="n">submodule_prefix</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="p">(</span><span class="s1">&#39;.&#39;</span> <span class="k">if</span> <span class="n">prefix</span> <span class="k">else</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">name</span>
                <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">named_modules</span><span class="p">(</span><span class="n">memo</span><span class="p">,</span> <span class="n">submodule_prefix</span><span class="p">):</span>
                    <span class="k">yield</span> <span class="n">m</span></div>

<div class="viewcode-block" id="Module.train"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.train">[docs]</a>    <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the module in training mode.</span>

<span class="sd">        This has any effect only on certain modules. See documentations of</span>
<span class="sd">        particular modules for details of their behaviors in training/evaluation</span>
<span class="sd">        mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,</span>
<span class="sd">        etc.</span>

<span class="sd">        Args:</span>
<span class="sd">            mode (bool): whether to set training mode (``True``) or evaluation</span>
<span class="sd">                         mode (``False``). Default: ``True``.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">training</span> <span class="o">=</span> <span class="n">mode</span>
        <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">children</span><span class="p">():</span>
            <span class="n">module</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">mode</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span></div>

<div class="viewcode-block" id="Module.eval"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.eval">[docs]</a>    <span class="k">def</span> <span class="nf">eval</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the module in evaluation mode.</span>

<span class="sd">        This has any effect only on certain modules. See documentations of</span>
<span class="sd">        particular modules for details of their behaviors in training/evaluation</span>
<span class="sd">        mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,</span>
<span class="sd">        etc.</span>

<span class="sd">        This is equivalent with :meth:`self.train(False) &lt;torch.nn.Module.train&gt;`.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span></div>

<div class="viewcode-block" id="Module.requires_grad_"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.requires_grad_">[docs]</a>    <span class="k">def</span> <span class="nf">requires_grad_</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Change if autograd should record operations on parameters in this</span>
<span class="sd">        module.</span>

<span class="sd">        This method sets the parameters&#39; :attr:`requires_grad` attributes</span>
<span class="sd">        in-place.</span>

<span class="sd">        This method is helpful for freezing part of the module for finetuning</span>
<span class="sd">        or training parts of a model individually (e.g., GAN training).</span>

<span class="sd">        Args:</span>
<span class="sd">            requires_grad (bool): whether autograd should record operations on</span>
<span class="sd">                                  parameters in this module. Default: ``True``.</span>

<span class="sd">        Returns:</span>
<span class="sd">            Module: self</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">():</span>
            <span class="n">p</span><span class="o">.</span><span class="n">requires_grad_</span><span class="p">(</span><span class="n">requires_grad</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span></div>

<div class="viewcode-block" id="Module.zero_grad"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.zero_grad">[docs]</a>    <span class="k">def</span> <span class="nf">zero_grad</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets gradients of all model parameters to zero.&quot;&quot;&quot;</span>
        <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">detach_</span><span class="p">()</span>
                <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">zero_</span><span class="p">()</span></div>

    <span class="k">def</span> <span class="nf">share_memory</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">t</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">share_memory_</span><span class="p">())</span>

    <span class="k">def</span> <span class="nf">_get_name</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>

<div class="viewcode-block" id="Module.extra_repr"><a class="viewcode-back" href="../../../../nn.html#torch.nn.Module.extra_repr">[docs]</a>    <span class="k">def</span> <span class="nf">extra_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Set the extra representation of the module</span>

<span class="sd">        To print customized extra information, you should reimplement</span>
<span class="sd">        this method in your own modules. Both single-line and multi-line</span>
<span class="sd">        strings are acceptable.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="s1">&#39;&#39;</span></div>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># We treat the extra repr like the sub-module, one item per line</span>
        <span class="n">extra_lines</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">extra_repr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_repr</span><span class="p">()</span>
        <span class="c1"># empty string will be split into list [&#39;&#39;]</span>
        <span class="k">if</span> <span class="n">extra_repr</span><span class="p">:</span>
            <span class="n">extra_lines</span> <span class="o">=</span> <span class="n">extra_repr</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
        <span class="n">child_lines</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">mod_str</span> <span class="o">=</span> <span class="nb">repr</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
            <span class="n">mod_str</span> <span class="o">=</span> <span class="n">_addindent</span><span class="p">(</span><span class="n">mod_str</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
            <span class="n">child_lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39;(&#39;</span> <span class="o">+</span> <span class="n">key</span> <span class="o">+</span> <span class="s1">&#39;): &#39;</span> <span class="o">+</span> <span class="n">mod_str</span><span class="p">)</span>
        <span class="n">lines</span> <span class="o">=</span> <span class="n">extra_lines</span> <span class="o">+</span> <span class="n">child_lines</span>

        <span class="n">main_str</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_name</span><span class="p">()</span> <span class="o">+</span> <span class="s1">&#39;(&#39;</span>
        <span class="k">if</span> <span class="n">lines</span><span class="p">:</span>
            <span class="c1"># simple one-liner info, which most builtin Modules will use</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">extra_lines</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">child_lines</span><span class="p">:</span>
                <span class="n">main_str</span> <span class="o">+=</span> <span class="n">extra_lines</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">main_str</span> <span class="o">+=</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">  &#39;</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">  &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span>

        <span class="n">main_str</span> <span class="o">+=</span> <span class="s1">&#39;)&#39;</span>
        <span class="k">return</span> <span class="n">main_str</span>

    <span class="k">def</span> <span class="fm">__dir__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">module_attrs</span> <span class="o">=</span> <span class="nb">dir</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span>
        <span class="n">attrs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
        <span class="n">parameters</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
        <span class="n">modules</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
        <span class="n">buffers</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
        <span class="n">keys</span> <span class="o">=</span> <span class="n">module_attrs</span> <span class="o">+</span> <span class="n">attrs</span> <span class="o">+</span> <span class="n">parameters</span> <span class="o">+</span> <span class="n">modules</span> <span class="o">+</span> <span class="n">buffers</span>

        <span class="c1"># Eliminate attrs that are not legal Python variable names</span>
        <span class="n">keys</span> <span class="o">=</span> <span class="p">[</span><span class="n">key</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">keys</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">key</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">isdigit</span><span class="p">()]</span>

        <span class="k">return</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">keys</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_replicate_for_data_parallel</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">replica</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
        <span class="n">replica</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="n">replica</span><span class="o">.</span><span class="n">_parameters</span> <span class="o">=</span> <span class="n">replica</span><span class="o">.</span><span class="n">_parameters</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="n">replica</span><span class="o">.</span><span class="n">_buffers</span> <span class="o">=</span> <span class="n">replica</span><span class="o">.</span><span class="n">_buffers</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="n">replica</span><span class="o">.</span><span class="n">_modules</span> <span class="o">=</span> <span class="n">replica</span><span class="o">.</span><span class="n">_modules</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="c1"># Warn users that gradients don&#39;t behave as expected on replica modules</span>
        <span class="n">old_zero_grad</span> <span class="o">=</span> <span class="n">replica</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="n">zero_grad</span>
        <span class="n">weak_self</span> <span class="o">=</span> <span class="n">weakref</span><span class="o">.</span><span class="n">ref</span><span class="p">(</span><span class="n">replica</span><span class="p">)</span>

        <span class="k">def</span> <span class="nf">zero_grad</span><span class="p">():</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                <span class="s2">&quot;Calling .zero_grad() from a module that was passed to a nn.DataParallel() has no effect. &quot;</span>
                <span class="s2">&quot;The parameters are copied (in a differentiable manner) from the original module. &quot;</span>
                <span class="s2">&quot;This means they are not leaf nodes in autograd and so don&#39;t accumulate gradients. &quot;</span>
                <span class="s2">&quot;If you need gradients in your forward method, consider using autograd.grad instead.&quot;</span><span class="p">)</span>
            <span class="n">replica</span> <span class="o">=</span> <span class="n">weak_self</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">replica</span><span class="p">:</span>
                <span class="n">old_zero_grad</span><span class="p">(</span><span class="n">replica</span><span class="p">)</span>

        <span class="n">replica</span><span class="o">.</span><span class="n">zero_grad</span> <span class="o">=</span> <span class="n">zero_grad</span>

        <span class="k">return</span> <span class="n">replica</span></div>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
         <script src="../../../../_static/jquery.js"></script>
         <script src="../../../../_static/underscore.js"></script>
         <script src="../../../../_static/doctools.js"></script>
         <script src="../../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>